{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use Elibitility Trace Actor-Critic to Play Acrobot-v1\n",
    "\n",
    "TensorFlow version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow.compat.v2 as tf\n",
    "tf.random.set_seed(0)\n",
    "from tensorflow import keras\n",
    "from tensorflow import nn\n",
    "from tensorflow import optimizers\n",
    "from tensorflow import losses\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import models\n",
    "\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14:32:46 [INFO] env: <AcrobotEnv<Acrobot-v1>>\n",
      "14:32:46 [INFO] action_space: Discrete(3)\n",
      "14:32:46 [INFO] observation_space: Box(-28.274333953857422, 28.274333953857422, (6,), float32)\n",
      "14:32:46 [INFO] reward_range: (-inf, inf)\n",
      "14:32:46 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 15}\n",
      "14:32:46 [INFO] _max_episode_steps: 500\n",
      "14:32:46 [INFO] _elapsed_steps: None\n",
      "14:32:46 [INFO] id: Acrobot-v1\n",
      "14:32:46 [INFO] entry_point: gym.envs.classic_control:AcrobotEnv\n",
      "14:32:46 [INFO] reward_threshold: -100.0\n",
      "14:32:46 [INFO] nondeterministic: False\n",
      "14:32:46 [INFO] max_episode_steps: 500\n",
      "14:32:46 [INFO] _kwargs: {}\n",
      "14:32:46 [INFO] _env_name: Acrobot\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('Acrobot-v1')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])\n",
    "for key in vars(env.spec):\n",
    "    logging.info('%s: %s', key, vars(env.spec)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ElibilityTraceActorCriticAgent:\n",
    "    def __init__(self, env):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.gamma = 0.99\n",
    "        self.actor_lambda = 0.9\n",
    "        self.critic_lambda = 0.9\n",
    "\n",
    "        self.actor_net = self.build_net(\n",
    "                input_size=env.observation_space.shape[0],\n",
    "                hidden_sizes=[100,],\n",
    "                output_size=self.action_n, output_activation=nn.softmax,\n",
    "                loss=losses.categorical_crossentropy, learning_rate=0.0001)\n",
    "        self.critic_net = self.build_net(\n",
    "                input_size=env.observation_space.shape[0],\n",
    "                hidden_sizes=[100,],\n",
    "                learning_rate=0.0002)\n",
    "\n",
    "    def build_net(self, input_size, hidden_sizes, output_size=1,\n",
    "                activation=nn.relu, output_activation=None,\n",
    "                loss=losses.mse, learning_rate=0.001):\n",
    "        model = keras.Sequential()\n",
    "        for layer, hidden_size in enumerate(hidden_sizes):\n",
    "            kwargs = {'input_shape': (input_size,)} if layer == 0 else {}\n",
    "            model.add(layers.Dense(units=hidden_size, activation=activation,\n",
    "                    **kwargs))\n",
    "        model.add(layers.Dense(units=output_size, activation=output_activation))\n",
    "        optimizer = optimizers.Adam(learning_rate)\n",
    "        model.compile(optimizer=optimizer, loss=loss)\n",
    "        return model\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "            self.discount = 1.\n",
    "            self.actor_trace_tensors = [0. * weight for weight in\n",
    "                    self.actor_net.get_weights()]\n",
    "            self.critic_trace_tensors = [0. * weight for weight in\n",
    "                    self.critic_net.get_weights()]\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        probs = self.actor_net.predict(observation[np.newaxis])[0]\n",
    "        action = np.random.choice(self.action_n, p=probs)\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "            if len(self.trajectory) >= 8:\n",
    "                self.learn()\n",
    "            self.discount *= self.gamma\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        pass\n",
    "\n",
    "    def learn(self):\n",
    "        state, _, _, action, next_state, reward, done, _ = self.trajectory[-8:]\n",
    "        states = state[np.newaxis]\n",
    "        q = self.critic_net.predict(states)[0, 0]\n",
    "        next_v = self.critic_net.predict(next_state[np.newaxis])[0, 0]\n",
    "        target = reward + (1. - done) * self.gamma * next_v\n",
    "        td_error = target - q\n",
    "\n",
    "        # train actor\n",
    "        state_tensor = tf.convert_to_tensor(states, dtype=tf.float32)\n",
    "        with tf.GradientTape() as tape:\n",
    "            pi_tensor = self.actor_net(state_tensor)[0, action]\n",
    "            logpi_tensor = tf.math.log(tf.clip_by_value(pi_tensor, 1e-6, 1.))\n",
    "        grad_tensors = tape.gradient(logpi_tensor, self.actor_net.variables)\n",
    "        self.actor_trace_tensors = [self.gamma * self.actor_lambda * trace +\n",
    "                self.discount * grad for trace, grad in\n",
    "                zip(self.actor_trace_tensors, grad_tensors)]\n",
    "        actor_grads = [-td_error * trace for trace in self.actor_trace_tensors]\n",
    "        actor_grads_and_vars = tuple(zip(actor_grads, self.actor_net.variables))\n",
    "        self.actor_net.optimizer.apply_gradients(actor_grads_and_vars)\n",
    "\n",
    "        # train critic\n",
    "        with tf.GradientTape() as tape:\n",
    "            v_tensor = self.critic_net(state_tensor)[0, 0]\n",
    "        grad_tensors = tape.gradient(v_tensor, self.critic_net.variables)\n",
    "        self.critic_trace_tensors = [self.gamma * self.critic_lambda * trace +\n",
    "                grad for trace, grad in\n",
    "                zip(self.critic_trace_tensors, grad_tensors)]\n",
    "        critic_grads = [-td_error * trace for trace in self.critic_trace_tensors]\n",
    "        critic_grads_and_vars = tuple(zip(critic_grads,\n",
    "                self.critic_net.variables))\n",
    "        self.critic_net.optimizer.apply_gradients(critic_grads_and_vars)\n",
    "\n",
    "\n",
    "agent = ElibilityTraceActorCriticAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14:32:48 [INFO] ==== train ====\n",
      "14:34:49 [DEBUG] train episode 0: reward = -481.00, steps = 482\n",
      "14:36:50 [DEBUG] train episode 1: reward = -500.00, steps = 500\n",
      "14:38:55 [DEBUG] train episode 2: reward = -500.00, steps = 500\n",
      "14:40:59 [DEBUG] train episode 3: reward = -500.00, steps = 500\n",
      "14:43:03 [DEBUG] train episode 4: reward = -500.00, steps = 500\n",
      "14:45:10 [DEBUG] train episode 5: reward = -500.00, steps = 500\n",
      "14:47:00 [DEBUG] train episode 6: reward = -427.00, steps = 428\n",
      "14:49:05 [DEBUG] train episode 7: reward = -500.00, steps = 500\n",
      "14:50:53 [DEBUG] train episode 8: reward = -438.00, steps = 439\n",
      "14:52:59 [DEBUG] train episode 9: reward = -500.00, steps = 500\n",
      "14:54:56 [DEBUG] train episode 10: reward = -482.00, steps = 483\n",
      "14:57:02 [DEBUG] train episode 11: reward = -500.00, steps = 500\n",
      "14:59:07 [DEBUG] train episode 12: reward = -500.00, steps = 500\n",
      "15:00:59 [DEBUG] train episode 13: reward = -459.00, steps = 460\n",
      "15:03:00 [DEBUG] train episode 14: reward = -500.00, steps = 500\n",
      "15:05:04 [DEBUG] train episode 15: reward = -500.00, steps = 500\n",
      "15:06:55 [DEBUG] train episode 16: reward = -445.00, steps = 446\n",
      "15:08:30 [DEBUG] train episode 17: reward = -399.00, steps = 400\n",
      "15:09:34 [DEBUG] train episode 18: reward = -285.00, steps = 286\n",
      "15:10:39 [DEBUG] train episode 19: reward = -299.00, steps = 300\n",
      "15:12:21 [DEBUG] train episode 20: reward = -500.00, steps = 500\n",
      "15:13:57 [DEBUG] train episode 21: reward = -472.00, steps = 473\n",
      "15:15:07 [DEBUG] train episode 22: reward = -334.00, steps = 335\n",
      "15:15:40 [DEBUG] train episode 23: reward = -160.00, steps = 161\n",
      "15:16:29 [DEBUG] train episode 24: reward = -244.00, steps = 245\n",
      "15:17:12 [DEBUG] train episode 25: reward = -214.00, steps = 215\n",
      "15:18:07 [DEBUG] train episode 26: reward = -269.00, steps = 270\n",
      "15:18:54 [DEBUG] train episode 27: reward = -233.00, steps = 234\n",
      "15:19:46 [DEBUG] train episode 28: reward = -256.00, steps = 257\n",
      "15:20:31 [DEBUG] train episode 29: reward = -219.00, steps = 220\n",
      "15:21:27 [DEBUG] train episode 30: reward = -265.00, steps = 266\n",
      "15:22:24 [DEBUG] train episode 31: reward = -268.00, steps = 269\n",
      "15:23:02 [DEBUG] train episode 32: reward = -189.00, steps = 190\n",
      "15:23:48 [DEBUG] train episode 33: reward = -229.00, steps = 230\n",
      "15:24:30 [DEBUG] train episode 34: reward = -209.00, steps = 210\n",
      "15:25:11 [DEBUG] train episode 35: reward = -206.00, steps = 207\n",
      "15:25:44 [DEBUG] train episode 36: reward = -162.00, steps = 163\n",
      "15:26:28 [DEBUG] train episode 37: reward = -219.00, steps = 220\n",
      "15:27:22 [DEBUG] train episode 38: reward = -270.00, steps = 271\n",
      "15:27:58 [DEBUG] train episode 39: reward = -179.00, steps = 180\n",
      "15:28:33 [DEBUG] train episode 40: reward = -175.00, steps = 176\n",
      "15:29:13 [DEBUG] train episode 41: reward = -201.00, steps = 202\n",
      "15:29:43 [DEBUG] train episode 42: reward = -143.00, steps = 144\n",
      "15:30:19 [DEBUG] train episode 43: reward = -174.00, steps = 175\n",
      "15:30:46 [DEBUG] train episode 44: reward = -133.00, steps = 134\n",
      "15:31:19 [DEBUG] train episode 45: reward = -159.00, steps = 160\n",
      "15:32:10 [DEBUG] train episode 46: reward = -237.00, steps = 238\n",
      "15:32:45 [DEBUG] train episode 47: reward = -171.00, steps = 172\n",
      "15:33:24 [DEBUG] train episode 48: reward = -172.00, steps = 173\n",
      "15:34:06 [DEBUG] train episode 49: reward = -199.00, steps = 200\n",
      "15:34:45 [DEBUG] train episode 50: reward = -165.00, steps = 166\n",
      "15:35:21 [DEBUG] train episode 51: reward = -162.00, steps = 163\n",
      "15:36:00 [DEBUG] train episode 52: reward = -184.00, steps = 185\n",
      "15:36:22 [DEBUG] train episode 53: reward = -103.00, steps = 104\n",
      "15:36:56 [DEBUG] train episode 54: reward = -173.00, steps = 174\n",
      "15:37:24 [DEBUG] train episode 55: reward = -140.00, steps = 141\n",
      "15:37:54 [DEBUG] train episode 56: reward = -150.00, steps = 151\n",
      "15:38:22 [DEBUG] train episode 57: reward = -142.00, steps = 143\n",
      "15:38:47 [DEBUG] train episode 58: reward = -124.00, steps = 125\n",
      "15:39:20 [DEBUG] train episode 59: reward = -164.00, steps = 165\n",
      "15:39:56 [DEBUG] train episode 60: reward = -162.00, steps = 163\n",
      "15:40:24 [DEBUG] train episode 61: reward = -126.00, steps = 127\n",
      "15:40:58 [DEBUG] train episode 62: reward = -153.00, steps = 154\n",
      "15:41:43 [DEBUG] train episode 63: reward = -203.00, steps = 204\n",
      "15:42:23 [DEBUG] train episode 64: reward = -179.00, steps = 180\n",
      "15:42:54 [DEBUG] train episode 65: reward = -141.00, steps = 142\n",
      "15:43:27 [DEBUG] train episode 66: reward = -150.00, steps = 151\n",
      "15:43:59 [DEBUG] train episode 67: reward = -139.00, steps = 140\n",
      "15:44:47 [DEBUG] train episode 68: reward = -219.00, steps = 220\n",
      "15:45:25 [DEBUG] train episode 69: reward = -178.00, steps = 179\n",
      "15:45:58 [DEBUG] train episode 70: reward = -159.00, steps = 160\n",
      "15:46:31 [DEBUG] train episode 71: reward = -154.00, steps = 155\n",
      "15:47:00 [DEBUG] train episode 72: reward = -140.00, steps = 141\n",
      "15:48:04 [DEBUG] train episode 73: reward = -299.00, steps = 300\n",
      "15:48:25 [DEBUG] train episode 74: reward = -98.00, steps = 99\n",
      "15:48:58 [DEBUG] train episode 75: reward = -159.00, steps = 160\n",
      "15:49:26 [DEBUG] train episode 76: reward = -134.00, steps = 135\n",
      "15:49:54 [DEBUG] train episode 77: reward = -125.00, steps = 126\n",
      "15:50:19 [DEBUG] train episode 78: reward = -118.00, steps = 119\n",
      "15:50:54 [DEBUG] train episode 79: reward = -163.00, steps = 164\n",
      "15:51:22 [DEBUG] train episode 80: reward = -138.00, steps = 139\n",
      "15:51:55 [DEBUG] train episode 81: reward = -160.00, steps = 161\n",
      "15:52:34 [DEBUG] train episode 82: reward = -191.00, steps = 192\n",
      "15:53:03 [DEBUG] train episode 83: reward = -142.00, steps = 143\n",
      "15:53:30 [DEBUG] train episode 84: reward = -129.00, steps = 130\n",
      "15:53:56 [DEBUG] train episode 85: reward = -126.00, steps = 127\n",
      "15:54:22 [DEBUG] train episode 86: reward = -131.00, steps = 132\n",
      "15:54:51 [DEBUG] train episode 87: reward = -140.00, steps = 141\n",
      "15:55:20 [DEBUG] train episode 88: reward = -140.00, steps = 141\n",
      "15:55:43 [DEBUG] train episode 89: reward = -112.00, steps = 113\n",
      "15:56:19 [DEBUG] train episode 90: reward = -174.00, steps = 175\n",
      "15:56:38 [DEBUG] train episode 91: reward = -94.00, steps = 95\n",
      "15:57:02 [DEBUG] train episode 92: reward = -116.00, steps = 117\n",
      "15:57:33 [DEBUG] train episode 93: reward = -147.00, steps = 148\n",
      "15:58:04 [DEBUG] train episode 94: reward = -151.00, steps = 152\n",
      "15:58:29 [DEBUG] train episode 95: reward = -126.00, steps = 127\n",
      "15:58:57 [DEBUG] train episode 96: reward = -133.00, steps = 134\n",
      "15:59:19 [DEBUG] train episode 97: reward = -114.00, steps = 115\n",
      "15:59:44 [DEBUG] train episode 98: reward = -122.00, steps = 123\n",
      "16:00:03 [DEBUG] train episode 99: reward = -88.00, steps = 89\n",
      "16:00:28 [DEBUG] train episode 100: reward = -122.00, steps = 123\n",
      "16:00:52 [DEBUG] train episode 101: reward = -119.00, steps = 120\n",
      "16:01:19 [DEBUG] train episode 102: reward = -133.00, steps = 134\n",
      "16:01:46 [DEBUG] train episode 103: reward = -133.00, steps = 134\n",
      "16:02:04 [DEBUG] train episode 104: reward = -87.00, steps = 88\n",
      "16:02:04 [INFO] ==== test ====\n",
      "16:02:10 [DEBUG] test episode 0: reward = -93.00, steps = 94\n",
      "16:02:18 [DEBUG] test episode 1: reward = -126.00, steps = 127\n",
      "16:02:25 [DEBUG] test episode 2: reward = -107.00, steps = 108\n",
      "16:02:31 [DEBUG] test episode 3: reward = -99.00, steps = 100\n",
      "16:02:39 [DEBUG] test episode 4: reward = -118.00, steps = 119\n",
      "16:02:47 [DEBUG] test episode 5: reward = -132.00, steps = 133\n",
      "16:02:55 [DEBUG] test episode 6: reward = -121.00, steps = 122\n",
      "16:03:02 [DEBUG] test episode 7: reward = -127.00, steps = 128\n",
      "16:03:08 [DEBUG] test episode 8: reward = -95.00, steps = 96\n",
      "16:03:16 [DEBUG] test episode 9: reward = -122.00, steps = 123\n",
      "16:03:27 [DEBUG] test episode 10: reward = -186.00, steps = 187\n",
      "16:03:39 [DEBUG] test episode 11: reward = -164.00, steps = 165\n",
      "16:03:47 [DEBUG] test episode 12: reward = -108.00, steps = 109\n",
      "16:03:54 [DEBUG] test episode 13: reward = -112.00, steps = 113\n",
      "16:04:05 [DEBUG] test episode 14: reward = -136.00, steps = 137\n",
      "16:04:13 [DEBUG] test episode 15: reward = -101.00, steps = 102\n",
      "16:04:20 [DEBUG] test episode 16: reward = -102.00, steps = 103\n",
      "16:04:26 [DEBUG] test episode 17: reward = -88.00, steps = 89\n",
      "16:04:32 [DEBUG] test episode 18: reward = -87.00, steps = 88\n",
      "16:04:40 [DEBUG] test episode 19: reward = -115.00, steps = 116\n",
      "16:04:49 [DEBUG] test episode 20: reward = -130.00, steps = 131\n",
      "16:04:58 [DEBUG] test episode 21: reward = -134.00, steps = 135\n",
      "16:05:07 [DEBUG] test episode 22: reward = -139.00, steps = 140\n",
      "16:05:16 [DEBUG] test episode 23: reward = -117.00, steps = 118\n",
      "16:05:23 [DEBUG] test episode 24: reward = -104.00, steps = 105\n",
      "16:05:36 [DEBUG] test episode 25: reward = -185.00, steps = 186\n",
      "16:05:44 [DEBUG] test episode 26: reward = -118.00, steps = 119\n",
      "16:05:52 [DEBUG] test episode 27: reward = -110.00, steps = 111\n",
      "16:06:01 [DEBUG] test episode 28: reward = -128.00, steps = 129\n",
      "16:06:07 [DEBUG] test episode 29: reward = -81.00, steps = 82\n",
      "16:06:15 [DEBUG] test episode 30: reward = -108.00, steps = 109\n",
      "16:06:22 [DEBUG] test episode 31: reward = -100.00, steps = 101\n",
      "16:06:29 [DEBUG] test episode 32: reward = -102.00, steps = 103\n",
      "16:06:36 [DEBUG] test episode 33: reward = -96.00, steps = 97\n",
      "16:06:42 [DEBUG] test episode 34: reward = -91.00, steps = 92\n",
      "16:06:52 [DEBUG] test episode 35: reward = -146.00, steps = 147\n",
      "16:07:00 [DEBUG] test episode 36: reward = -119.00, steps = 120\n",
      "16:07:07 [DEBUG] test episode 37: reward = -99.00, steps = 100\n",
      "16:07:14 [DEBUG] test episode 38: reward = -103.00, steps = 104\n",
      "16:07:21 [DEBUG] test episode 39: reward = -98.00, steps = 99\n",
      "16:07:30 [DEBUG] test episode 40: reward = -122.00, steps = 123\n",
      "16:07:36 [DEBUG] test episode 41: reward = -91.00, steps = 92\n",
      "16:07:44 [DEBUG] test episode 42: reward = -120.00, steps = 121\n",
      "16:07:53 [DEBUG] test episode 43: reward = -121.00, steps = 122\n",
      "16:08:01 [DEBUG] test episode 44: reward = -116.00, steps = 117\n",
      "16:08:07 [DEBUG] test episode 45: reward = -114.00, steps = 115\n",
      "16:08:12 [DEBUG] test episode 46: reward = -114.00, steps = 115\n",
      "16:08:18 [DEBUG] test episode 47: reward = -122.00, steps = 123\n",
      "16:08:24 [DEBUG] test episode 48: reward = -121.00, steps = 122\n",
      "16:08:28 [DEBUG] test episode 49: reward = -102.00, steps = 103\n",
      "16:08:36 [DEBUG] test episode 50: reward = -185.00, steps = 186\n",
      "16:08:41 [DEBUG] test episode 51: reward = -111.00, steps = 112\n",
      "16:08:47 [DEBUG] test episode 52: reward = -135.00, steps = 136\n",
      "16:08:53 [DEBUG] test episode 53: reward = -121.00, steps = 122\n",
      "16:08:59 [DEBUG] test episode 54: reward = -135.00, steps = 136\n",
      "16:09:03 [DEBUG] test episode 55: reward = -98.00, steps = 99\n",
      "16:09:09 [DEBUG] test episode 56: reward = -144.00, steps = 145\n",
      "16:09:15 [DEBUG] test episode 57: reward = -117.00, steps = 118\n",
      "16:09:22 [DEBUG] test episode 58: reward = -163.00, steps = 164\n",
      "16:09:26 [DEBUG] test episode 59: reward = -92.00, steps = 93\n",
      "16:09:31 [DEBUG] test episode 60: reward = -126.00, steps = 127\n",
      "16:09:38 [DEBUG] test episode 61: reward = -153.00, steps = 154\n",
      "16:09:44 [DEBUG] test episode 62: reward = -132.00, steps = 133\n",
      "16:09:48 [DEBUG] test episode 63: reward = -95.00, steps = 96\n",
      "16:09:54 [DEBUG] test episode 64: reward = -126.00, steps = 127\n",
      "16:09:59 [DEBUG] test episode 65: reward = -112.00, steps = 113\n",
      "16:10:03 [DEBUG] test episode 66: reward = -102.00, steps = 103\n",
      "16:10:08 [DEBUG] test episode 67: reward = -115.00, steps = 116\n",
      "16:10:13 [DEBUG] test episode 68: reward = -110.00, steps = 111\n",
      "16:10:18 [DEBUG] test episode 69: reward = -126.00, steps = 127\n",
      "16:10:24 [DEBUG] test episode 70: reward = -121.00, steps = 122\n",
      "16:10:28 [DEBUG] test episode 71: reward = -96.00, steps = 97\n",
      "16:10:34 [DEBUG] test episode 72: reward = -137.00, steps = 138\n",
      "16:10:40 [DEBUG] test episode 73: reward = -138.00, steps = 139\n",
      "16:10:46 [DEBUG] test episode 74: reward = -136.00, steps = 137\n",
      "16:10:50 [DEBUG] test episode 75: reward = -91.00, steps = 92\n",
      "16:10:55 [DEBUG] test episode 76: reward = -115.00, steps = 116\n",
      "16:11:06 [DEBUG] test episode 77: reward = -260.00, steps = 261\n",
      "16:11:12 [DEBUG] test episode 78: reward = -138.00, steps = 139\n",
      "16:11:16 [DEBUG] test episode 79: reward = -87.00, steps = 88\n",
      "16:11:21 [DEBUG] test episode 80: reward = -116.00, steps = 117\n",
      "16:11:26 [DEBUG] test episode 81: reward = -116.00, steps = 117\n",
      "16:11:31 [DEBUG] test episode 82: reward = -112.00, steps = 113\n",
      "16:11:36 [DEBUG] test episode 83: reward = -106.00, steps = 107\n",
      "16:11:41 [DEBUG] test episode 84: reward = -134.00, steps = 135\n",
      "16:11:45 [DEBUG] test episode 85: reward = -79.00, steps = 80\n",
      "16:11:50 [DEBUG] test episode 86: reward = -112.00, steps = 113\n",
      "16:11:55 [DEBUG] test episode 87: reward = -122.00, steps = 123\n",
      "16:12:01 [DEBUG] test episode 88: reward = -141.00, steps = 142\n",
      "16:12:05 [DEBUG] test episode 89: reward = -100.00, steps = 101\n",
      "16:12:11 [DEBUG] test episode 90: reward = -136.00, steps = 137\n",
      "16:12:16 [DEBUG] test episode 91: reward = -106.00, steps = 107\n",
      "16:12:20 [DEBUG] test episode 92: reward = -100.00, steps = 101\n",
      "16:12:24 [DEBUG] test episode 93: reward = -95.00, steps = 96\n",
      "16:12:29 [DEBUG] test episode 94: reward = -95.00, steps = 96\n",
      "16:12:37 [DEBUG] test episode 95: reward = -206.00, steps = 207\n",
      "16:12:42 [DEBUG] test episode 96: reward = -111.00, steps = 112\n",
      "16:12:47 [DEBUG] test episode 97: reward = -112.00, steps = 113\n",
      "16:12:58 [DEBUG] test episode 98: reward = -249.00, steps = 250\n",
      "16:13:02 [DEBUG] test episode 99: reward = -104.00, steps = 105\n",
      "16:13:02 [INFO] average episode reward = -120.59 ± 29.65\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD4CAYAAAAEhuazAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABE20lEQVR4nO29eXhkZ3Xn/3lrL5X2tSV1q/e2u93e2zsYzBIbMNhkIQYSMyzxQCBD5gkhY8hkMpMfSZ5AliFhGULAkABmx8bBBpvFNt7bWy/ufVNLrX0tVan29/fHXapKtaiqVepWqc7nefRIulW3dK+u9L2nvue85yitNYIgCEJt4TjfByAIgiCce0T8BUEQahARf0EQhBpExF8QBKEGEfEXBEGoQVzn+wBKpb29XW/YsOF8H4YgCEJV8fzzz49rrTsWbq8a8d+wYQO7d+8+34chCIJQVSilTuXbLraPIAhCDSLiLwiCUIOI+AuCINQgIv6CIAg1iIi/IAhCDSLiLwiCUIOI+AuCINQgIv6CIAgrlOdOTvLZnx8hHEtU/LVF/AVBEJaR2Uic6/7m5zx6eKzsfR87PMY/PXIYt7PyUi3iLwiCsIzsG5xhaCbCr4+UL/79k2F6mv0i/oIgCMtBMqUJRStvrQAcHAoan4eDZe/bPxmmr7Wu0ocELFH8lVK/o5Tar5RKKaV2LXjsbqXUUaXUIaXUzRnbr1RK7TUf+6xSSi3lGARBEJbKPU+e5DWf/iWReLLir31weBaAA0Pli//plSr+wD7gN4HHMjcqpXYAdwAXAbcAn1dKOc2HvwDcBWw1P25Z4jEIgnAeOT0ZPt+HsGSOj80xPhdj98mpir+2Jfrjc1HGgtGS9wtFE4zPxVi3EsVfa31Aa30oz0O3AfdqraNa6xPAUeBqpVQ30Ki1fkobk+O/Dty+lGMQBOH8sWdgmlf/3S/Zf2bmfB/KkpgOxwF47Cx8+WIkkikOjwTZ0d0IpN8FlMLpKeOmulIj/0L0Aqczvh8wt/WaXy/cnhel1F1Kqd1Kqd1jY5W9KIIgLJ3RWSOSPTEeOs9HsjQmQzHAqK6pJCcnwkQTKW6/vAdI+/+l0D9xnsVfKfWIUmpfno/biu2WZ5susj0vWusvaa13aa13dXTkzCIQhBXFV359gs/98uj5PoxzSiyZAmBktnQ7YyUyFTbE/+BwkJHZSMVe14r0r9/cTmeDlwNlRP79pp22vu08ib/W+g1a6515Pu4rstsAsC7j+7XAGXP72jzbBaHq+fGeM/zoxcHzfRjnlGjCSJCOBisnmOeDqXCMS9Y2AZxVPX4hDg4FcToUWzrrubC7sazI//RkmAafiya/u2LHk8ly2T73A3copbxKqY0Yid1ntdZDQFApda1Z5XMnUOwmIghVw3Q4zoRpH9QK0bgR+Y9WceSvtWYqHOfaTW10Nngrav0cHJ5lU3sAn9vJ9jUNHB2dI26+W1oMq8xzuQoil1rq+Xal1ABwHfCfSqmfAmit9wPfAV4BHgI+rLW2aqg+BHwZIwl8DHhwKccgCCuFqXCMqXCMRIn/3KuBaMKyfao38g/HksQSKVoDHl69tYNfHx0nmSroRpfFgaEgF5rJ3gu7G4glUyXnR5azxh+WXu3zQ631Wq21V2vdpbW+OeOxT2mtN2utL9BaP5ixfbdpG23WWn/ErPoRhKommdLMzMfRGibDtRP9x0zxHy2jhPF8MhuJ8x9PnyJTdqxkb0udmxu3tTMdjrN3cOnVS7OROIPT81y4pgGAC9cYN4EDQ4v7/qmU5vTU/MoVf0EQDCzhB5iYqx3xtzz/aon8v/3saf78R/s4NjZnb7PKPFvqjMhfKXj00NlZP9945hT7zBvHIXNF7/ZuQ/w3d9TjcqiSVvqOBqPEEqllq/EHEX9BqAhTGdF+bYm/EfkHIwnmY5VfHVtpXjxtLOLKrE6y3qm1Bjy0Bjxc3Nt0VvX+QzPzfPKH+3jvPc8xGoxw0IzwrYjf43KwpbPe3l4Mq9JHIn9BWOFMZ4j/+Fx1WCCVwBJ/KK3i5/EjY3zhV8eW85CK8mL/NJD9TsW6ds11HsAoy9wzMG2/qymVn+0fAWAmHOe/fetF9g3O0uhz0d3ks5+zvbvRjvy11gXfMYn4C0KVMBmK21/XlPhn9MJZrNb/hf4pPvC13XzmZ4dKrnipJEMz8wzNGGKbmaOwPP/WgCH+F/c2EU9qDg/P5b5IER7cN8TWznr+5jcv5unjk3zvhQG2dzdmVetcuKaBoZkIP3pxkNs/9wTX/HX+Vs/9k2EcCnqa/WWfZ6mI+AtCBciyfWqo3DOWIeLFfP9TEyE+8LXdJFKaZEozODV/Lg4vi5fMqB+yS1OnwnGUwq6nv7jXqPcvJ+k7MRfl2ROTvGnnGn7ryrW88+p1JFOa7Walj4VV+fPH336JiVCMtoCHLz9+POf1Tk+G6W7y43Etn0SL+AtCBbCsgwavi/EyK18i8SQf+eYLVdkfJxpP0eBzAYUrfqZCMf7LV58jpTV//fadAJw6D83gXjw9jcfloLfZn2VRTYViNPndOB1GhL6u1U+jz8W+Mq7HIwdGSGm4eecaAP7XWy/ijqvWcdtlPVnPu2pDC795RS+f/u1L+OXHXsv7XrWRx4+Mc3gkOwm83GWeIOIvCBVhMhTH5VD0tdWVHfn/8uAoD+wZ4qemZ5yP/okwv/9vz2TlFlYC0USKjgYvHpeD0TyRv9aaP/3eHgan5vnXO3fx2gs6gbSnvRSSKc19Lw3yji8+xbef61/0+S/2T7Gzp5HeFn9W5D8ZjtFi+v0ASil29jaxv0jk/7cPHuQzPz1kl4w+uG+Yda1+u4Gbz+3kb3/rEi7va8nar87j4h/ecRm/s2sdbqeDd13dh9fl4KtPnMh6noi/IFQJ0+EYLQEP7fVeJsr0/B/YMwQYbYUL8YuDIzx+ZJzHj4wv6TgrTTSRxOdy0tXozWv7/OCFQR45MMLHb7mAqza00lHvxety0D+xtEZwP9k7xBv/4VE+eu9LHBye5c++v5e/vH9/wQV28WSKPQMzXN7XQmeDNyvynw7HaKnLbqGws7eJA8PBvLmJeDLF1548yb/88ij/+vhxZiNxnjg6zpt2dpe9Grcl4OE3r1jLD14YtHMP87EkY8EofcvU08dCxF8QKsCUKSBt9R7Gyyj1DMcS/PygEfEfHyssiIdHjRvD86cq329+KUQTKbxuB50NvpyE75npef7yx/u5ekMr771hIwAOh6KvtY5TE2cf+R8ZCfKH33gBj8vBF3/vCp7/n2/k/a/ayD1PnuTOrzzLbCSes8+h4SDRRIrL+5rpbPAtSPjG7WSvxc7eJmKJFEdGcm/IewZmmI8nWd9Wx1//5CCf+MFe4knNzRetOavzed8NG4gmUnzrWePdi9XKeTlr/EHEX6gBHthzhqllTsJOheI013noqPcyPhel1IXrPz8wSiSeYmdvIyfGQ6QKtBU4bJYHvtC/8sTf43TQ1ZgdTWut+bPv7yGZ0nz6dy6x/XQwulQuxfb54qPH8budfPMPruWWnd24nQ7+5607+PRvX8KzJyb5ix/ty9nnRfP3dnlfC12NXsKxJHPm2MbpcMwu87TY2WPYN/vyWD/PnpgE4Jt/cC271rfwwJ4huhq9XL6u+azOZ2tXA6/e2s49T57ksz8/whcfNUphxfYRhCUwPBPhI998ka8s8FQrzVQ4Rmudh7Z6D9FEilCJC54e2HOGzgYvv7trHfPxJMMFfPPDI0GUgv1nZgnHlmfW7NlgRP5OI5rOiPz/c+8Qjx8Z5xNv3s76tkDWPn2tAfonwyXfIDMZnJ7nvpcGuePqdTnR+u/sWsdHXreFH710hgf3DmU99mL/NJ0NXnqafHQ2eoF0ddJkKJbzWhvaAtR78yd9nzkxwZbOenqb/fzrnbu4dF0zd163AYfj7Buwfei1m5kOx/iHhw/zgxcGaQt42NwRWHzHJeBa1lcXhPOM5aM/d3JyWX/OVDhOS8BNW8AQlvFglHpv8X+vYCTOLw+N8a6r+9jcWW8ebyintntkNspsJMFrL+jgV4fG2DMww7Wb2pbnRMokGk/ibfDS2eglGE0QjiWo87j41aExWurcvOvqvpx91rfVEY4lGZ+L0dFg/L5SKY2GrHcI+fjXx4yyyA+8elPexz980xZ+fmCUT/5oH1dtbKW93nj9F09Pc3lfM0opOhuMRVejs1F6mvxEEymaF3j+DodiR09jTrlnIpli98kp3mZW8bQEPNz34RsW+S0tzvWb2zn0V2+yh5so8xiWE4n8hVXNcbOD4ov903YTskqjtbatg3ZTzCZC6Sg4Ek/mTUQ+cmCEWCLFWy/tZnOHIf7H8iR9rTLAO64yRmQsxfd/aN8wvzhYuKqoGOFYIidajyVSeF0OujIEFYzo+OqNrXkFzLIz+ifTOY6Pfe9l3nvPc0V//mQoxr3P9XPbZb30Flj85HY6+Pt3XMpcNMHdP9jLyfEQ+wZnODEe4rJ1RuVNp3mNRoORdGuHBbYPGPX+B4Zms67dgaEgc9EE12xsLXqsZ4PDoXCaH8st/CDiL6xyrCRqNJEqq267HILRBImUNmwf0z7ITPq+/fNP8pFvvpiz3wMvD9HT5OPydUYFSsDjzFvxY4n/VRta2dwR4IUSxP/p4xN2lGwxNDPPH3/7Rf75F+VPGzsxHuKKv3qYXxwczdoeTaTwuBx0NRriPzIb4cz0PKcn57lmY/53J1YVi5X0TaU0Pz8wyq+PjBUtZb3nyZNE4ik++Jr8Ub/Ftq4GPvYb23j4lRFe+5lfces//xqAK9eb4m8e61gwaueCFnr+ADt7G4nEUxzLSMQ/c2ICoOC5VRNi+wirhi89doyLe5u5bnP6H/PE+BxrGn0Mz0bYfXKSKxbUXVeCabO1Q3Od27YxrBYPs5E4B4ZmOTA0y8OvjPDGHV2AkUh89PAY73vVRjvK29RRb79TyeTwSJD2eg9t9V6uXN/Cz14ZQWtdtKzwK78+wc9eGaGn2c9bLukG4O9/dphIPHVWye97njhBJJ5icDp7ZW40kcLrcqZ99GDUbqFwdYHoeG2LH6XS4n9wOMjMvPE7/PXRcW69pCfvft96tp83bO9ka1fDosf7gVdtYmtXg30zqfe6uWqDce0bfS68LgcjsxF7ZfZCzx/SK333Dc5wgdmW+ZkTk6xvq2NNRr+eakUif2FVoLXmMz87nJPYPTEe4soNLWxoq+O5k8tTKWMJSEudx14sZHX2tMb2+dwO/vL+/YRjCWbCcT70jefpaPDywddstl9nU0cgb7nnoZE5tpmCd+X6FqbD8bw3iUyOmu8g/vxHexkLRtl/ZobvvzCAx+VgKpxbClmMmfk4331+ADAsrEyiieQC2yfCMycmaPC5clobWHhdTnqa/Jw2K36eNaNpr8tRcIpWMBJnLBjlyvWl2S0Oh+KmCzp5++Vrefvla3njji77ZqmUorPRy2gwmtHXJ3dU4sb2euo8Ttv3T6U0z52cXBbL53wg4i+sCqbDcWKJVNaqzFgixempeTa1B7hqQyu7T04WLKVcCpZv3BLw4HE5aPK77YVe1uCOv/vtSxmcnueff3GUP/nuywxNR/iXd12RFXFuaq9ncHo+qzVyKqU5OhLMEn8o7vvHEilOTYS55aI1hGJJ7v7BXv76Jwdo9rt59zV9zMzHy5o29p3nThM2j2k+lr2fVeff6HcZq3yDUZ45McnVG1qLJm/XtfrtFg/PnJikt9nP67d38tjh8bxVQNY7jrUtlWl0ZlUnWb3889k+TodiR3cjjx8ZYywY5dBIkOlwnKtXgeUDIv7CKsEqkTwzE7GF9/RUmGRKs9EU/6lwnOPj5XVqLIVpO/I3osf2jIVer5yZpaXOzVsv6ea3rljLF351jEcOjPCJN2+3hdxik1nalznmb3B6nlAsydYuIyG8qb2eJr+7qO9/aiJEMqW5ZecaPn7zBTxyYIQnjk7w0ddvZYNZdmnZLIuRSKa458mTXLOxFY/TwXxG5K+1NhO+TpRSdDV62X9mhuNjoYKWj8X61gCnJoxyz2dPTHLNplZu3NrB8GyEI6O512hgsrLib61LsCL/5gJD0t9z/QZOT87zxn98lH98+DCARP6CsJLIrI/fd8aIti0LZWN7gF2m37sc1s9UKD0JCqDNXOgFcGB41m7r+4k3X0h7vZdbL+nmvTdsyHkdq+In8wZ1ZNSwjS4wI3+HQ3FFX3PRyN8Szy2d9bz3ho1cv7mNC7oaeNc16+2SxqkSewQ9/MoIg9PzvO9VG/G5HVm2j9XL32t2nuxq8PHUMTMhukgpal9bHeNzUfYMzDARinHNxlZu3NYBkNf6GTBXva5tqczCp3TkH6PR58LlzC+Fb720h5989FVsag8YOZQmX8VuQOcbSfgKq4KRmQzxH5zhNds6OGGK6Mb2AE1+N+31Hp47Ock789SeL4WpcAyHgkZ/OvI/NBwkkUxxcDjIndeuB4ybwuMfvwmf25E3Wbux3YjKj42mI/9DZk/5zCTnletb+OWhMWbm43Yb4kyOmuK/qSOA06H49/dfQzxpVOVYNlMpvn8qpfnyr0+wrtXPG7Z34fc4sywpq52zJf6djV5SGuo8Ti7qye/3W6w3K36+Z+YSrtnYRk+zny2d9Tx6eCynjn9gah6vy0F7fa49czZ0NBjrEganI3mTvZls6Wzgux+8nu/sPk1bwFN2/56VikT+wqpgeDaCUtDb7LeX5J8YD9Ea8NBcZ/zD7lrfyu7liPzD2S2B2+u9TIRinBgPEUukshKffo+zoHj4PU56m/3Zkf9IkDWNviyRt24Epwo0Rzs6Okdvs586jxHbOR0Kn9sJpN+dTC5S8ROKJvjwN1/g+VNT3PXqTTgdCr/bmWX7ROMLxN9M+l65vgV3gUjawqr1v++lQTobvPbN4DXbOnj2xGROYnlgat6sEqqM8Fq1/odHgnn9/oU4HYp3Xt3Hb5xl/56ViIi/sCoYmY3QFvByWV+zXZ1xfCxkR9MAuza00D8Zrviw8alwPKslcFvAy3Q4zp4B4zh2LBIFZ7Kw4ufQSND2+y2sBU6FBqIcHZ1jS2d93sdazCi3WD396ckwv/WFJ/np/mH+/C3b+T3znYvP7Vxg+xhfe13GjcUq9yxl9fH6VuO6zEYSXLOpzRb1G7d1EE2keOZE9orsgelwxSwfwF6X0D8ZXjTyX62I+NcQpyZC3PfS4Pk+jGVheCbCmiYvO3uaGJiaZzpsRN6bssTfSNSVskiqHKx2zhZtpjXxxNFx3E5le/mlsKk9wPGxObQ2Jl4dHZ2z/X6LdaYILqy5B6PH/bGxObYWEP9WO/LPb/vEkyl+9/89xeD0PF9979V84NWbbGH2LYz8Lc/fbchIT5NxUyolIdpU57bfzWQ+/5qNrXhdDh49lO37W5F/pbBuVEDWjbuWEPGvIe597jT//dsvkVyGcselcGoixNeePLmk1xiejbKm0WcvzHn6+CSjwSgbM5pjWVZDvuZppZJIpvjhiwNZw0MmQ/GsfvBWP5nHj46ztbOhrFF8mzrqCcWS9E+G+cELA0QTKbvM06LR76Le62IgT+Q/ODVPNJEqGPn7PU68LkfBhO/Txyc4MxPh0799Ca8xE7D2vgsif6tdhse0eG7ZuYYvvPuKnCqmQlhWT6b4+9xOrlzfktWLKRiJMx2OVzTytywqIKeXf60gCd8aYj6WJKVhdj6eFameb775bD//79Hj3H5ZL01n+Y84Mhvhir5mO9H44z1nALIi/ya/G6U4qxWuiWSK778wwOd+ecwern3LRd001bmZDseyEpxWUnIsGOXGrR2FXjIvVrnn6/7+UZIpTXu9h2s2ZUfSSil6m/15xf/omFEdVEj8wVjNWuh38NC+Yfxupz1xKxO/x8loMP2OYWHk73M7edPF3cVOL4vNHfWcmY7kHOvFa5v46q9PEjNbR1S6xh8MwXc7FfGkXlH/C+cSifxrCCtqK7XM71xxatwo4zvbiDyaSDIZirGm0UdLwMPaFj8/P2A0L9vYnhYWp0PR7Hfbi7LK4atPnOTPvr+X5jo3f/LGbaQ0PHHMmKo1Fc5uCdxWn7YUtncv3oogk0vXNXPdpjbeefU6vvmBa3j67tfntEQGQwjz2T5HM8o8C9Fc58n7N5BMaX66f4SbLuywE8SZ+N3Z1T7ReLbnXy53v+lCvvUH1+QkcXf2NBFLpuyeRlZuo5Lin9ndU2wfYdWTFv/ylvcvN9ZKz6GZ/AnMxbA6SXaZ/VZ29jQRiadQKm0tWLQEPHZdfjk8d3KSTe0B7vvwDXzotZtp8Ll47PAY87EkkXh2S+DMcsRykr0AjT4337rrWv6/2y/m+i3tBevPe1v8DE7lDkQ5OjpHe72naAVLa8Cd92/ghf4pxuei3LIzf/RuJHzTK3wX1vmXS2ejL2+fHsu6swbaD9jiX9nhJlYfpnytHWoBEf8awvrHnZlfOZG/1touWTzbKhzrHcMas4Lj4rWGePQ0+XMi2NY6T06ZYyplrDI9MR4qmA85OBxke4+xWMvldHDD5nYeOzyW1dfHot7rsn3+HQX62yyV3mY/s5FEzsjCYpU+Fs11+W2fh/YN43E6uOmC/FaV31NokdfZRf6F6Guto8Hrsqu2BqbCFa3xt7DKPWs18hfPv4aImKV5ZxP5LhfjczG7b8zQzFmKv7mf1WnR8t835ZmE1BLw2A3FLJ46PsG7v/wMAB6Xg509jXz9/dfYw1jmogn6J8O8Y9dae58bt3Xw0P5hOzGZKSBKKTrqvWitS6ohPxt6W9Llno3dRuSqtebI6By3XZa/K6ZFax7bR2vNQ/uGefXWdhp8+SNhnyu72sdO+J5l5F8Ih0NxUW8j+waNldqVrvG3sCp+xPMXVj0r0fPPXKhUauR/YGiW/owB4NZ+Vu32TtM2yKzxt2gL5Eb+VuuAT7z5Qm6/rIcX+qd5LqPO/JA5P/fCNeko/sZt7QDc95KRWF5YMbKls37RFgdLwbJAMmv9x4JRgpEEWxYpLW0JeJiej2e9y9k3OMvg9Dw37yy8iMnvMcTfaryWrvOvvIzs7EkPUjHEv/LzbK1OpAuneNUKEvnXEGnbZ+VE/idNEW/wuRaN/EPRBJ/+6SG+9tRJLl/XzA/+0BifNzwTwe920ugz/pzb6738z1t38Oqt7Tmv0RIwot7MfvhjQSNncOd1G9Aavv/CIM+fmuKmC42Kl4PDRgR6YUbydm1LHZs7AnYfmoXR45fuvLK8X0SZWAu9BjJ8/3Syt3iSuaXOjdbG34GVqH5w3xBOh+KN27sK7udzO9HasHt8bmdOtU8luXhtE9FEiqNjcwxMhbnEtPIqye2X9+JxOejISNDXEhL51xArMfLvnwjhUHBFX4tt3+Rjz8A0N//TY9zz5Ek2tAXYMzBjV54Mz0ZY0+TLsgXe/6qNOfXxYFge8aRmLpoegj4WjNLgc+FzO/GbfWkyG6cdHArS4HXljA68cVsHCTN6Xugbe13OinvhmbTXe/BmlEFCuof/Yp5/ur9P+u/g4VdGuGZja1ELxG/mT6y/o6VW+xTjoh5D7J85PslUhWv8Lda11vFfX7N51fTqKRcR/xrCitSmV1C1z8mJMD3Nfta1+ovaPv/0yBHmY0m++8Hr+Itbd5BIaV48bQj0yGyErsbSojdL3DLzHpmDxMG4Eb10etrueX9weJYLuxtyROLGjEVQ59o6UEoZFT8Z4r93YIbmOveivwsrD2ElfaOJJMfG5uwV0IXweyzxT5n7LY/nD8b6jIDHyUP7hoHKlnkKBiL+NYQVsa0k8T81GWZDW4DuJj9T4XhOQy+LiVCMHT2NXLWhlSv6WlAKu0nb8GzErvRZDKusL7PWfywYzXrrf+X6FubjSQ4OB9Fac3AomOX3W1y7sQ2Py0GD17VoI7PlYOFCr6eOT3DNxtZFI1mrxYNV7tk/ESalsxfE5cOK/K2kb2yJpZ7FcDgUO3oa7Zm5Iv6VR8S/hliJts+piRDr2+qyBoDnYzajfXFTnZsLuhp47uQkWmtGZqJ2jf9itCyIegHG5qK0N2SLPxjTsgam5glGE1l+v4Xf4+Saja1Z7xrOJWtb/HbC9/RkmIGpea4rIcls9/Q3fwfWSMh8CfJMrLJZy26LJlI4FLiKTOxaChf1NGHlpJfD9ql1JOFbQ1hv11dK5D8TNnq2rG+ro9sU76GZSN4VrQt71+/a0MIPXxhkbC5KLJkqI/LPbWm8MPLvafbT3eTj+VNT9Jg+f77IH+Cv337xeUugr22pYyIUYz6W5KnjRoR83ebcJPdCFnr+1uSwDYuKvxErWpG/Mb+3cIvqpWIt9lqOGn9BIv+aQWtt1/kXa+dbCaKJJP/7x/vtcYqFODVpiM76tkDRyF9rnSP+V21oJRRL8iuz+2Op4t+yQPjmY0nmoomc6P2K9S08f2qKg+YM3gvW5K+gWddaZ5eWnmvs1s7T8zx9bIK2gIdtXYt3EK3zOPE4Hbb1dWIsRHu9J+9gmExyEr7m/N7lwvq9LkeNvyDiXzPEkim0Nv6BQ7Gk7dcuB/vPzPLVJ07y8CsjRZ9nlXmub6uzF2jlK/eciyZIpnRWUvUqMzn5wJ4hgJJtnwavC5dD2ZG/NW5xYbnflX0tDE7P86vDY/S11tkLvlYS1kKvgakwTx6b4NrNbSWJpFKKloCbaTPpfXx8jk3ti980rISvbfvEU3ZHz+Vgc0cAn9shls8yIeJfI1iWj2WvTC9jiwfLSz42VnxYer+5wMsS1wavK2+5p2WrZEamPc1+epv9PHHUaK5WauRvCF96heuoWeO/MPLP9P0vLBD1n2+sJOiTxyYYno2U5PdbtNR50pH/eGhRvx8yIn/zHWQsubyRv8vp4I9et5XfvWrdsv2MWmZJV04p9Wml1EGl1B6l1A+VUs0Zj92tlDqqlDqklLo5Y/uVSqm95mOfVfJ+7pxg1WRb9spy+v5WVJ05kSofJyfCdDV67XGDXU2+ksUfDN8/mdIolSvexcjs72NH/gv239HTaHvcFy5Tf56l0tngw+VQ/OAFY0DPdZvLE/+pUIyZ+Tjjc7GsuQeFyE34Jpd1LQPAh2/awpvLaBMtlM5Sb9sPAzu11pcAh4G7AZRSO4A7gIuAW4DPK6Wsv5IvAHcBW82PW5Z4DEIJLIz8z6anfalYUbVVRVKI/omwPc7POrZ8bZ0t8W/MEX/D+mmv95ZVatkScNt1/mMFIn+308Ela5sB2L5CI3+nQ9Hd7GN8Lkpng3fRUs1MWs13PydLrPSBzDr/tO2zHGWewrlhSVdOa/0zrbW1VPJpwOp8dRtwr9Y6qrU+ARwFrlZKdQONWuuntNEg5OvA7Us5BqE0rLfqa2zbZzkjf7N+fDJcNLdw0izztOhqzB/5z5rH2uzPrvi4aoNhzZRq+Vi0BtKWhyX++ea4WtZPoWTvSsBK+l5Xot9v0VxntHW2Kn1KuXH4FtT5RxMi/tVMJbNY7wO+bX7di3EzsBgwt8XNrxduz4tS6i6Mdwn09fVV8FBrDytas8V/GSt+rNdOpjT9k+G87QbCsQSjwWiW+Hc3+Ribi5JIprL62Nu2z4JVtNs6G2j0uWwrq1RaMloaj81FaQ148r5z+P1r19Na5ykpKj5fGMnQSa4vw/IB42Y3HY5xdHQOh4K+tsWTqj5T6Odj1grf5LKs7hXODYuKv1LqESBfq79Paq3vM5/zSSABfMPaLc/zdZHtedFafwn4EsCuXbtW1uDZKsOyfdacI8/f6VAkU5rjY/n7y/dPWpU+aWHtavSRTGnG52L2TSrzWBd6/g6H4p/uuIyO+vIj/6lwjFRKM76gxj+TnmY/f3DjprJe+1xjDXO/btPi9f2ZNNd5SGl4eWCatS11JXn3LqcDj9ORtcJ3uVpWC8vPouKvtX5DsceVUu8BbgVer61er0ZEn5miXwucMbevzbNdWGasyL814MHjdCzrNK+pcIzt3Q3sG5wt6Pu/cGoagA1t2Z4/pBu1WczMx3E6FAFPrkC97sLCXSgL0WIK38x83FzdW70C9s5r1rGhva6kyD0Tq83FS/3TXFHiwHUwFnpFxPZZFSy12ucW4M+At2mtMydk3A/coZTyKqU2YiR2n9VaDwFBpdS1ZpXPncB9SzkGoTSsf1if22kPHV8uJkMx1rcGaK/3cmw0u9zzmeMTvPvLT/OJH+6lt9mf9a7Asm8W+v7WAq9KFYa1matFJ8OxnNW91UZng4/bLivonBbEitiD0URZtpbf41ywyGt5q32E5WOpnv+/AF7gYfMf82mt9Qe11vuVUt8BXsGwgz6stbY6dn0IuAfwAw+aH8IyEzETrz63g5Y697LaPlPhOM11bjZ3BLIi/18dGuW/fPU52uu9fPLN23n3tX12BQlkRP4LZvnOzMdpXmT1aTlY/X0mQzHG56LnrTfP+aQ1w67JN/GsEH53eppXNJ6UyL+KWZL4a623FHnsU8Cn8mzfDexcys8VyieS0Xu9Oc8Yv0qRSmmmwzFaAx42ddTz0L4h+7H7Xz5Dc52bxz9+U5boW1iW1PBsdluImfl4TpnnUrAqe05PhonEU7RXceR/tmTOHygn8ve5nXadfyyZkoRvFSNXrkaIZtg+zf7syP+hfcP8+1MnK/JzZiNxUtoQl80dAabCcaZCMRLJFL84OMrrLujMK/xgrL7tavLmjfwX6ztTDlZ/n0MjxnjGWoz8WwLp3+emRcY+ZuLLivzF869mVl7DEmFZsKp9DNvHw8sD0/ZjX3z0GMMzEX7/ug1L/jnWytnWgIdGv/HndXx8jkRSMx2O84YdxRO0axpzF3rNzMezEsNLxbI8Dg/XrvjXe124nQqHUnSXUSrrdy/w/Jd5ha+wfIj41wiZCV9rgY/WmkRK84o5KHthfX0hjo4G6Wr00eDLjcYtO6kl4GF9q1GBcmwsxJGRIB6nI2v6VT7WNPnZm3FjgspH/n6PE5/bweERIxldi+KvlKK5zkNbwIOjjH78fo+TsWCCVEobvX0k8q9a5MrVCJFEEqdD4XY6aK7zEEukmI8nOTwSJJZIkdLGgqfFSCRT3P65J/n8r47lfdxa3dtS52Ztix+3U3F8LMTDr4xw7ea2Rbtjdjf5GJqJYFUNp1Ka2fl4xcckttZ57BGItej5g7Gq91KzhUWpWAnfWHL5hrcL5waJ/GuESDxlr9BsMYV0Ohxnz8CM/Zwz0xG6m4qPy+ufDDMXTXB0NH/HTjvyr/PgcjrY0BbgkQMjnJwI8/5XL75gqqfJRzSRYiIUo73ey1wsQUrnLvBaKi0BD2dmIjgdKmf4eq1wz3uvxlGmdlsJX3t+73kYXylUBrlyNUIknrR7s9hj/MKxLPHP11dnIZZVcnoynPfxqQzPH4wyQutG8YbtnYu+vjU564wZlc+E8zd1WyrW8bUFPDiXaQzhSsfvcZbt2fs9xiKvqNkrSur8qxcR/xohEk9liL8hfDPhOHsHp+1xeUMLqmzycXTUSJL2T4ZJL+hOMxmO4XE5qDMreqxKkp29jYu+q4D0gBJrNm2hds5LxYr2a9XyOVt8LsP2icaXb3i7cG6QK1cjRBJJ25+1Iv+RYIRDw0Fu2NKO3+3MO0VrIUfMKD4cSzKRpy30VChGa53HXo1rdYt84/Z87aFyWdtsJIktP365xN+K/Gsx2bsU/B5T/K3IX8S/apErVyNE40l85lt8K+p96tgE8aTm0rVNdDf7Sor8D4/M2f/w/Xmsn8lQdnL2mo1tbGoP8LbLeko6zka/i4DHmSP+lU74Wr8DEf/y8LmdaA3BiNHJXcS/epErVyMYto9xua0o+rHDxgjEi9c22VU2mTyw5wyPHxmzv0+mNMfG5uz2wfl8f2t1r0VfWx2/+NhrS15FqpSit8W/7LaP1dhMxL88rFGO1nWROv/qRcS/RshM+PrcTvxuJ8OzEdoCHnqb/XQ3+Rmazhb/v3rgFf7mJwft70+bw1led6GRuO2fyBP5h2P2CtqzpbfZv+y2j3WM4vmXh7U6Oy3+IiHVily5GiGSSIs/pMs9L17bhFKK7iYfo8EICbN+eyYcZ2Q2yitDs0yY9f+HzXYIF69tprPBm9f2sTz/pdCTIf7T4Thup7IjzkrRKrbPWWFdB6s9iNT5Vy9y5WqETNsHoMkUv0vMSp/uJj8pDaPmWMPDZlUPwBPHJoB0sndLZz19rXU54p9Maabn40uP/Fv8TIfjhKKJirdztrh4bRNvubibaze1VvR1Vzu+heIvtk/VIuJfI0QyEr6QGfk3A+l2ypbvb0X5bqfiiSNGbuDo6Bw9TT7qvS76WutyPP+Z+Thap1/7bOnNqPWfrXBrB4sGn5vPvfsKOhvKmwJW61gBhGX7SFfP6kWuXI0QiWcP3rCqXS5Za0b+zdmDVA4PBwl4nNx0QSe/PjqO1prDI0G2dhnDzPva6hiajdglf5Dd1G0pWOI/MD1f8b4+wtLITfiKhFQrcuVqhGg8mWX7bO4IsKWz3p6e1d1oCK5V7nl4ZI6tXQ28ems7g9PzHB8PcXR0jq3m5K2+1jq0Ti/GgvTg9qW2S7AWep0R8V9xpBO+xrUW26d6EfGvERYmfD/6hm088Eevsr9v9Luo86QXeh0ZDbKtq54bthiDwb/93GmiiRRbu9LiD9m1/pWK/DsbfLgcisGpeabnYyL+K4ichK9E/lWLNHarAZIpTTypszx/p0PhdKS/V0qxpslY6DUxF2V8Lsa2rgY2tgfoafJx77P9AGnbxxT/TN8/s53zUnA6jGMZnJ5nJiyR/0rCTvjOS7VPtSNXrgZI9/Ivfrl7mvwMzUTs5m3buhpQSnHDlnZmzRWd1sD1jgYvXpdjQeSfbue8VHqb/ZyeDBOMJuzKJOH8s7DOX7p6Vi9y5WqAzEEuxVjT5GNoOmJX+mwzo/xXbTWsnzWNPhrNAS5KqZxyz6lwDK/LUZGa/N4WP4dH5tDL0M5ZOHusv6GZcBynQ5U0/EdYmciVqwEiifQIx2L0mAu9DgzN0uhz0dVoLIC6frMh/pbfb2GIfzrhOxUyWjtUoia/t9nPXNR4tyHiv3KwZkLIFK/qR65eDVB65G8s9Pr10XHb8gHD4nnHrrW89ZLs5mzrzFp/q7XzVDhWscEoVrkniPivJFxOh231iPhXN5LwrQEs8V+sLM+q9R+Yms+Ztft3v31pzvP7WuuYiyaYCsdpDXiYDMWWXOljYZV7goj/SsPndpiRv5R5VjNy664BIvHSbB9rlS/Ats76Is80WFjuORWu3KzdzMi/0u2chaVhJX1ldW91I1evBoiWaPtkTtratqZh0dftazPE/9RECKCikX+P2D4rFiuhL7ZPdSO2Tw0QSZQm/o0+Y6FXOJa0K32Ksa6ljgafi0/+cB8Hh4PMRuIV8/x9bift9R7G52SR10rD+juSGv/qRq5eDVCq7WO1dm4LeErqc+/3OPn+h67ntRd08MVHj1WkqVsmvc1+PC7Hojct4dxii794/lWNRP41gF3tU8I/687eJuJmT/9S2NbVwL+86wr+eDTID18c5M0Xd5/1cS6kt8Vf0lxh4dxi2T6ywKu6EfGvAdKR/+Li/4/vuAx9Fj9jS2cDf3rzhWexZ2H+8LVbGJjKHRgjnF+shK/YPtWNiH8NUGp7BwCHo7JDU5bCzt4mdprDZoSVgyR8Vwdy9WqAUhO+glAK4vmvDkT8awDL9pFITagEfo+s8F0NyNWrAaLxJF6Xo+JzcIXaxCockEVe1Y1cvRogEk+K5SNUDDvhK7ZPVSPiX2VEE0n+9Lsv26tqSyEST5WU7BWEUpBFXqsDuXpVxrHREN99foD/+/MjJe+zcISjICwFqfZZHcjVqzKsHvc/fvkMo8HSFkBF4smSFngJQimI7bM6EPGvMkKm+MeTmm883V/SPmL7CJXEXuErkX9VI1evygia4r+pPcA3njllL+AqRiSexCu2j1AhfGL7rAqWdPWUUn+llNqjlHpJKfUzpVRPxmN3K6WOKqUOKaVuzth+pVJqr/nYZ5XUH5bFnDlI/Q9v2sL4XIwfv3xm0X0iiZR4/kLFsN5FivhXN0u9ep/WWl+itb4MeAD4CwCl1A7gDuAi4Bbg80opS32+ANwFbDU/blniMdQUc9E4ADdf1MW2rnq++sRJe4xiIaLxpD17VRCWip3wlYCiqlmSImitZzO+DYDdE+w24F6tdVRrfQI4ClytlOoGGrXWT2lDsb4O3L6UY6g15qJJlIKAx8V7b9jIK0Oz7B2cKbqP1PkLlaTBZ7TtDnjkb6qaWXJjN6XUp4A7gRngJnNzL/B0xtMGzG1x8+uF2wu99l0Y7xLo6+tb6qGuCuYiCQIeFw6HYkd3IwBjwWjRfSThK1SS7d0NfO5dV+TMeRaqi0UVQSn1iFJqX56P2wC01p/UWq8DvgF8xNotz0vpItvzorX+ktZ6l9Z6V0eH/KGBYfvUe417trXIJpoo3n9f6vyFSqKU4i2XdOOWfv5VzaKRv9b6DSW+1jeB/wT+F0ZEvy7jsbXAGXP72jzbhRIJRZPU+0zxN+usY4uJv9g+giAsYKnVPlszvn0bcND8+n7gDqWUVym1ESOx+6zWeggIKqWuNat87gTuW8ox1BrBaIKAFfm7rMi/cLmn1tqwfSThKwhCBkv1/P9WKXUBkAJOAR8E0FrvV0p9B3gFSAAf1lpbCvUh4B7ADzxofgglMheJ02CKv8e1uO1jPSaVGYIgZLIk8dda/1aRxz4FfCrP9t3AzqX83FpmLpqgs8EHZET+8SLiX8YIR0EQagfxAqqMfJ5/MdsnPcVLLrUgCGlEEaqMYCRd7eN2KpQqnvC15/dKEy5BEDIQ8a8itNbMRRO2+Cul8LocRT3/iNg+giDkQcS/ipiPJ0lpbNsHwONcTPzF9hEEIRdRhCrC6uVvRf5gVPEU9fxt8ZfIXxCENCL+VYTV0TNL/F2OotU+kYRl+8ilFgQhjShCFZE38nc5iCYXt31k6pIgCJmI+FcRduSf6fm7nMUjf7F9BEHIg4h/FVEw8i/i+acXecmlFgQhjShCFVFY/It5/hL5C4KQi4h/FWGLv29htY/YPoIglIeIfxURLFDtU2yFbzhmiL9fxF8QhAxE/KuIUDSBy6GyBmd7FvH852NJfG4HTke+OTqCINQqIv5VxFw0Qb3PhTEKwWCxOv9QLEGdZ8nTOgVBWGWI+FcRc5FEluUDRv1+Mc8/HEtSJ4O2BUFYgIh/FZHZ1M3C8PyL2z4i/oIgLETEv4rIK/7u4qWeoVgSv9g+giAsQMS/irA8/0y8ZldPrXXefeZjCQIS+QuCsAAR/yoir+dvlnDGCvT3Ec9fEIR8iPhXEYU8fyg8xD0sto8gCHkQ8a8iiol/oYVeYbF9BEHIg4h/lZBMacKxZI7n7ykp8hfxFwQhGxH/KiFfUzdI9+mPxnPLPbXW4vkLgpAXEf8qIVRQ/AtH/rFkimRKywpfQRByEPGvEvJ19ASjzh/yi/+82dRNIn9BEBYi4l8l5OvoCWnbJ1/CNyTiLwhCAUT8q4RCnn864Zvr+c/HjH3E9hEEYSEi/lVCqJDtY4l/ns6eoahE/oIg5EfEv0qYW8T2yef524NcRPwFQViAiH+VEDQj/wavO2u7t5jtEzf2CYjtIwjCAkT8qwTL9gl4s6N4q9onb8JXbB9BEAog4l8lzEUT+NwOXM7sS+ZxLl7qKbaPIAgLEfGvEoKRBPULLB9Id/XMZ/uEY2L7CIKQHxH/KmEumqDBlyviRat9JPIXBKEAIv5VQiiayPH7AVwOhUPl7+c/H0viUOkbhCAIgoWoQpWQb5ALgFIKjyv/KMdwLEnA40IpdS4OURCEKkLEv0oIRvN7/mDU+ufr6hmOJcTyEQQhLyL+VcJcNE59HtsHDFunUOQvZZ6CIORDxL9KCEVzB7lYeN3FxF8qfQRByKUi4q+U+phSSiul2jO23a2UOqqUOqSUujlj+5VKqb3mY59VYkiXxFyBUk8wbJ98i7zCsYRE/oIg5GXJ4q+UWge8EejP2LYDuAO4CLgF+LxSylKhLwB3AVvNj1uWegyrnWgiSSyZylvqCcZCr/x1/jLCURCE/FQi8v9H4OOAzth2G3Cv1jqqtT4BHAWuVkp1A41a66e01hr4OnB7BY5hVTM7n7+pm0Uh22ferPYRBEFYyJLEXyn1NmBQa/3ygod6gdMZ3w+Y23rNrxduL/T6dymldiuldo+NjS3lUKuawel5AHqb/Xkf97ocBRZ5ie0jCEJ+Fg0LlVKPAGvyPPRJ4BPAb+TbLc82XWR7XrTWXwK+BLBr166Cz1vt9E+GAehrq8v7uNflZDocy9k+L7aPIAgFWFT8tdZvyLddKXUxsBF42czZrgVeUEpdjRHRr8t4+lrgjLl9bZ7tQhFOm+K/rqWQ+Oe3fUKxBIECVpEgCLXNWds+Wuu9WutOrfUGrfUGDGG/Qms9DNwP3KGU8iqlNmIkdp/VWg8BQaXUtWaVz53AfUs/jdVN/0SYjgZvwSje43LkVPukUppIPIXfLZG/IAi5LEtYqLXer5T6DvAKkAA+rLW2ylE+BNwD+IEHzQ+hCP2TYfpa80f9YK7wXSD+83Hp5S8IQmEqJv5m9J/5/aeAT+V53m5gZ6V+bi3QPxnm6o2tBR83qn2ySz1D1vB2sX0EQciDrPBd4cQSKYZm5llXNPLPrfaxBrnUie0jCEIeRPxXOGem50lpFrd9FrR0toa3i+0jCEI+RPxXOHaZZxHxtxK+xro5g7DYPoIgFEHEf4VTivjb07wykr4S+QuCUAwR/xXO6ckwHpeDzgZvwecUE38p9RQEIR8i/iuc/skw61r8OByFm59aQ9xjWeJvDm8X20cQhDyI+JfAxFyUP/rWi8zMx8/5z16sxh/A67Qi/3S5p9g+giAUQ8S/BJ47OcmPXz7Dy6enz+nP1VrTP1GC+LtzbZ95EX9BEIog4l8C43NG07SJUPSc/tyZ+TjBaKJojT9keP4Ztf6hqCX+YvsIgpCLiH8JTIZM8Z/L7Zy5nJRS6QNGnT8ssH3iCTwuB84iuQJBEGoXEf8SmJgzIv6J0HkS/wKtnC2syD+2wPYJiOUjCEIBRPxLwBL9yQWR/3Q4xn0vDS7bz+1fpJWzhSdPqWcoKsPbBUEojIh/CUzYnn+2+H/v+QE+eu9LDM9EluXnnp4M017vWbRcM237ZET+8YQMchEEoSAi/iVge/4LEr6W6A/PLo/490+GF032Qma1T3app9g+giAUQsS/BCzRn1wQ+Y8Gje2jyyj+iyV7IX+1TzgqIxwFQSiMiP8ipFKaqbCxuGthtc9oMGJ+rnwJaCKZ4sx0ZFG/H9K2Tyyjs2c4nhDPXxCEgoj4L8LMfJxkStNe72EumiAST1srduS/DOI/GoySTGl6W/yLPtdO+GYcWzialAVegiAURMR/ESzLZ2tnA5Bt/YzOGo+NBStv+wzNzAOwpsm36HMLNXYT8RcEoRAi/otgWT3buuqBtPiHYwnmokbztJHZykf+Q2YyufusxV9sH0EQCiPivwhWeefWroas70czBH90OSL/aUv8F7d9XE5jJe/Cah+J/AVBKISI/yJYYr/NEv+5bJ+/t9mfdSOoFEMzEeo8Thp9pUXvHqfDXuEbS6RIpLSIvyAIBRHxXwRL7Ld0Zts+I2Z5587eRsbnjORsOewbnOHwSLDg40Mz83Q3+VCqtN48XrfDtn2sjp5+sX0EQSiAiP8iTIZiNPndtNS5cTuV3eHTivx39jSR0umbRKl87Lsv8xf37Sv4+NBMpCTLx8Lrcth1/iFrkItE/oIgFEDEfxEm5mK01XtQStEa8DAZsmyfCB6ng61mIriccs9kSnN8PMSxsVDB51iRf6l4XU7b87dHOIr4C4JQABH/RZgIRWkLeABoDXjt6p+x2SgdDV66Gg2BLifpe2Z6nlgixVgwSjCSOx0snkwxGoyWKf4Oe5FXepCL2D6CIORHxH8RJuZitAWM4ent9R47ATwSjNDZ6KXTEv8ykr4nxtMR/8nxcM7jo8EoWkN3c+m2j0dsH0EQykDEfxEmQzFa663I32MnfEdno3Q2eOmoN24M5dT6Z4r/8fG5nMeHy1jgZeF15Uv4ivgLgpAfEf8iJFOaqXCMdtP2aQt4s0o9Oxt8eFwOWurcZdk+x8fm8LudKJV9I7CwFnj1lJXwzfX8xfYRBKEQog5FmA7HSGkj4gdoq/cQiiWZCceZmY/T1WhE/V2NvrISvsfHQ2zprGcqHMsv/uYCr7Iif7eDUMiwe6bCxruT+hLXCAiCUHtI5F8Ey+JpM60dK/F7cHgWgM4GQ5w7Grxlif+J8RAb2wNsbA9wPE/Fz9BMhEAZC7zATPiats/egRma69z0lHHzEAShthDxL4JV05+u9jE+HxgyxL/DjPw7G3wl9/SPxJMMTs+zqSPA5o56ToyH0Dp7gdjQzDxryljgBeBxOW3P/8XTU1y2rrms/QVBqC1qQvwze96UgxX5WwnftnpL/I2VuZ0Npvg3ehkLRkmVsMq3fzKM1tiR/1w0wdiCBWLlLvACa5FXkmAkzpHROS5f11LW/oIg1BarWvy11tz+uSf4y/v3n9X+Vjtnq9TT+mzZPlaNf1eDl4SZHAYYnJ7nyaPjeV/z+JhR3bOpvZ6N7QEATiywfspd4AXpap89AzNoDZf3NZe1vyAItcWqFn+lFGtb/Pxs/0jZvXcg3c65pc4NpN8BHBoJ4nIoWuuM7+1af9P3/z8/3s97vvosM+HcBVzHzQTvhva6tPhnJH3PZoEXWNU+KV7snwLg0nXNZe0vCEJtsarFH+CWnWuYCMV47uRk2ftOhKK01LlxOY1fU4PXhcfpIBJP0V7vxeEwPHXL/hmZjRCMxPnloTHiSc1P9w/nvOaJsRCdDV4afG56mv14XI4s8T+bBV5gLPKKJVK82D/Nls56mvzuss9XEITaYdWL/00XdOJxOXhoX64QL8ZkKGYneQG7vw8YPr+FVfUzGozyyIERYokUfreTH+85k/OaVqUPgNOh2NBWZ78bgLNb4AXp9g4v9E9xuUT9giAswqoX/4DXxY1bO/jp/uGcqprFGJ+L2WWeFlbS1xJ8SN8IxoJRHnh5iO4mH++5fgNPHpvI6fZ5YjzEpo6A/b1R7ple5XtmuvwFXmDU+QNMheNc3ifJXkEQirPqxR/gTTvXMDQT4eWBmbL2mwzF7DJPi3yRv89t1OQfGQny2JEx3nJxN2+9tJtkSvNQhvUzE44zEYrZkT/Apo56+ifDJMymbMMz5S/wAsPzt5BkryAIi1ET4v/67Z24HKps62diLmpH+hbt9VZtf/Y7gs5GHz/ZN0w8qbn10h52dDeyqT3AAy8P2c+x+vhsaq+3t21sDxBPaganDbvnzMx82Qu8ID3Ht87jtKeOCYIgFGJJ4q+U+kul1KBS6iXz480Zj92tlDqqlDqklLo5Y/uVSqm95mOfVedgJVJznYfrNrfx0L6hkq2fZEozPR+nNZAt8nbk35AdmXc2eIklUqxr9XPp2iaUUtx6STfPnJiw+/5Yid2NGbbPJvNdgOX7D89Eyl7gBUbCF+CStU04HbK4SxCE4lQi8v9HrfVl5sdPAJRSO4A7gIuAW4DPK6UsX+ILwF3AVvPjlgocw6LcsnMNJyfCHCoyOjGTqXAMrSlo+3Q1Zt8UrJr/t1zcYwv3rZf2kNLw4F7jHceJ8RBOh2JdS52938Ja/zMzEXrKrPSBdOQvfr8gCKWwXJ2/bgPu1VpHgRNKqaPA1Uqpk0Cj1vopAKXU14HbgQeX6Ths3rijiz//0T7e85VnafQtXgYZNz341sBC26dw5A9w6yXd9rZtXQ1s66rnMz87xH88fYqR2QjrWvx2lG69fqPPxT//4gjferafkxMhbr+st+zzszx/qfQRBKEUKiH+H1FK3QnsBv5Eaz0F9AJPZzxnwNwWN79euD0vSqm7MN4l0NfXt6SD7Gzw8fGbL2Tv4HTJ+1yxvoXrN7dlbXv99i4++Jow27uzffW3X9FLwOviop7GrO13v2k7333+NABbu+p53YVdWY8rpfiT37iAZ05MAMYN43evWlfyMVpct7mNu27cxI3bOsreVxCE2kMt5oErpR4B1uR56JMYAj8OaOCvgG6t9fuUUp8DntJa/4f5Gv8G/AToB/5Ga/0Gc/urgY9rrd+62IHu2rVL7969u+QTEwRBEEAp9bzWetfC7YtG/pZQl/AD/hV4wPx2AMgMX9cCZ8zta/NsFwRBEM4hS6326c749u3APvPr+4E7lFJepdRGjMTus1rrISColLrWrPK5E7hvKccgCIIglM9SPf+/U0pdhmH7nAT+K4DWer9S6jvAK0AC+LDW2uqr/CHgHsCPkehd9mSvIAiCkM2inv9KQTx/QRCE8ink+dfECl9BEAQhGxF/QRCEGkTEXxAEoQYR8RcEQahBqibhq5QaA06d5e7tGIvRVju1cp4g57oaqZXzhHN7ruu11jlL/6tG/JeCUmp3vmz3aqNWzhPkXFcjtXKesDLOVWwfQRCEGkTEXxAEoQapFfH/0vk+gHNErZwnyLmuRmrlPGEFnGtNeP6CIAhCNrUS+QuCIAgZiPgLgiDUIKta/JVSt5gD5I8qpf7H+T6eSqKUWqeU+qVS6oBSar9S6qPm9lal1MNKqSPm51Ux1Fcp5VRKvaiUesD8frWeZ7NS6ntKqYPmtb1uNZ6rUuq/m3+3+5RS31JK+VbLeSqlvqKUGlVK7cvYVvDclFJ3mxp1SCl187k6zlUr/ubA+M8BbwJ2AO80B8uvFhIYYzO3A9cCHzbP738AP9dabwV+bn6/GvgocCDj+9V6nv8XeEhrfSFwKcY5r6pzVUr1Av8N2KW13gk4gTtYPed5D3DLgm15z838n70DuMjc5/Omdi07q1b8gauBo1rr41rrGHAvxmD5VYHWekhr/YL5dRBDJHoxzvFr5tO+Btx+Xg6wgiil1gJvAb6csXk1nmcjcCPwbwBa65jWeppVeK4Ys0T8SikXUIcx0W9VnKfW+jFgcsHmQud2G3Cv1jqqtT4BHMXQrmVnNYt/L3A64/uiw+KrGaXUBuBy4Bmgy5yYhvm58zweWqX4J+DjQCpj22o8z03AGPBV0+L6slIqwCo7V631IPAZjJneQ8CM1vpnrLLzXEChcztvOrWaxV/l2bbq6lqVUvXA94E/1lrPnu/jqTRKqVuBUa318+f7WM4BLuAK4Ata68uBENVrfRTE9LtvAzYCPUBAKfV75/eozhvnTadWs/gXGiK/alBKuTGE/xta6x+Ym0es2crm59HzdXwV4gbgbUqpkxjW3euUUv/B6jtPMP5mB7TWz5jffw/jZrDazvUNwAmt9ZjWOg78ALie1XeemRQ6t/OmU6tZ/J8DtiqlNiqlPBhJlfvP8zFVDKWUwvCGD2it/yHjofuB95hfvwe471wfWyXRWt+ttV6rtd6AcQ1/obX+PVbZeQJorYeB00qpC8xNr8eYg73azrUfuFYpVWf+Hb8eI2e12s4zk0Lndj9wh1LKq5TaCGwFnj0nR6S1XrUfwJuBw8Ax4JPn+3gqfG6vwnh7uAd4yfx4M9CGUU1wxPzcer6PtYLn/FrgAfPrVXmewGXAbvO6/ghoWY3nCvxv4CCwD/h3wLtazhP4FkYuI44R2b+/2LkBnzQ16hDwpnN1nNLeQRAEoQZZzbaPIAiCUAARf0EQhBpExF8QBKEGEfEXBEGoQUT8BUEQahARf0EQhBpExF8QBKEG+f8BzVmBRnboNRAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    episode_reward, elapsed_steps = play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > -120:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
